{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from datetime import datetime\n",
    "import json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 基础版本\n",
    "txt文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"./data/unstructured_example_log.txt\") as f:\n",
    "    txt = f.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['[5/15/2013 2:17:26 PM] Session Start\\n',\n",
       " '[5/15/2013 2:17:26 PM] Leaving sequence: loadXML, moving forward.\\n',\n",
       " '[5/15/2013 2:17:30 PM] Player submitted name: Carl\\n',\n",
       " '[5/15/2013 2:17:30 PM] Leaving sequence: InputNameScreen, moving forward.\\n',\n",
       " '[5/15/2013 2:17:31 PM] Player submitted name: Carl\\n']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "txt[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['[5/15/2013 2:17:26 PM] Session Start',\n",
       " '[5/15/2013 2:17:26 PM] Leaving sequence: loadXML, moving forward.',\n",
       " '[5/15/2013 2:17:30 PM] Player submitted name: Carl',\n",
       " '[5/15/2013 2:17:30 PM] Leaving sequence: InputNameScreen, moving forward.',\n",
       " '[5/15/2013 2:17:31 PM] Player submitted name: Carl']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 去掉\\n\n",
    "txt = [i.strip() for i in txt]\n",
    "txt[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'5/15/2013 2:17:26 PM'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 将时间字符串提取出来\n",
    "s_0 = txt[0].split(\"]\")[0].strip(\"[\")\n",
    "s_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "datetime.datetime(2013, 5, 15, 14, 17, 26)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 将时间字符串转化为datetime\n",
    "date_fmt = \"%m/%d/%Y %I:%M:%S %p\" # %H 代表二十四小时制 %I代表十二小时\n",
    "s_0 = datetime.strptime(s_0,date_fmt)\n",
    "s_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 组合起来\n",
    "def trans(event):\n",
    "    s = event.split(\"]\")[1].split(\":\")\n",
    "    s1 = s[0]\n",
    "    if len(s) >1 :\n",
    "        \n",
    "        s2 = s[1].lstrip()\n",
    "    else:\n",
    "        s2 = np.nan\n",
    "    return [s1,s2]\n",
    "df = pd.DataFrame(txt,columns=[\"原始数据\"])\n",
    "df[\"time\"] = df[\"原始数据\"].apply(lambda x: x.split(\"]\")[0].strip(\"[\"))\n",
    "df[\"time\"] = df[\"time\"].apply(lambda x:datetime.strptime(x,date_fmt))\n",
    "df[\"event_name\"] = df[\"原始数据\"].apply(lambda x:trans(x)[0])\n",
    "df[\"event_attribute\"] = df[\"原始数据\"].apply(lambda x:trans(x)[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>原始数据</th>\n",
       "      <th>time</th>\n",
       "      <th>event_name</th>\n",
       "      <th>event_attribute</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[5/15/2013 2:17:26 PM] Session Start</td>\n",
       "      <td>2013-05-15 14:17:26</td>\n",
       "      <td>Session Start</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[5/15/2013 2:17:26 PM] Leaving sequence: loadX...</td>\n",
       "      <td>2013-05-15 14:17:26</td>\n",
       "      <td>Leaving sequence</td>\n",
       "      <td>loadXML, moving forward.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[5/15/2013 2:17:30 PM] Player submitted name: ...</td>\n",
       "      <td>2013-05-15 14:17:30</td>\n",
       "      <td>Player submitted name</td>\n",
       "      <td>Carl</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[5/15/2013 2:17:30 PM] Leaving sequence: Input...</td>\n",
       "      <td>2013-05-15 14:17:30</td>\n",
       "      <td>Leaving sequence</td>\n",
       "      <td>InputNameScreen, moving forward.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[5/15/2013 2:17:31 PM] Player submitted name: ...</td>\n",
       "      <td>2013-05-15 14:17:31</td>\n",
       "      <td>Player submitted name</td>\n",
       "      <td>Carl</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                原始数据                time  \\\n",
       "0               [5/15/2013 2:17:26 PM] Session Start 2013-05-15 14:17:26   \n",
       "1  [5/15/2013 2:17:26 PM] Leaving sequence: loadX... 2013-05-15 14:17:26   \n",
       "2  [5/15/2013 2:17:30 PM] Player submitted name: ... 2013-05-15 14:17:30   \n",
       "3  [5/15/2013 2:17:30 PM] Leaving sequence: Input... 2013-05-15 14:17:30   \n",
       "4  [5/15/2013 2:17:31 PM] Player submitted name: ... 2013-05-15 14:17:31   \n",
       "\n",
       "               event_name                   event_attribute  \n",
       "0           Session Start                               NaN  \n",
       "1        Leaving sequence          loadXML, moving forward.  \n",
       "2   Player submitted name                              Carl  \n",
       "3        Leaving sequence  InputNameScreen, moving forward.  \n",
       "4   Player submitted name                              Carl  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "json文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./data/structured_example_log.json') as f:\n",
    "    txt = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'data': [{'session_time': '2013-05-15 14:17:26',\n",
       "   'event_name': 'Session Start',\n",
       "   'event_attribute': 'NaN'},\n",
       "  {'session_time': '2013-05-15 14:17:26',\n",
       "   'event_name': 'Leaving sequence',\n",
       "   'event_attribute': 'loadXML, moving forward.'},\n",
       "  {'session_time': '2013-05-15 14:17:30',\n",
       "   'event_name': 'Player submitted name',\n",
       "   'event_attribute': 'Carl'},\n",
       "  {'session_time': '2013-05-15 14:17:30',\n",
       "   'event_name': 'Leaving sequence',\n",
       "   'event_attribute': 'InputNameScreen, moving forward.'},\n",
       "  {'session_time': '2013-05-15 14:17:31',\n",
       "   'event_name': 'Player submitted name',\n",
       "   'event_attribute': 'Carl'},\n",
       "  {'session_time': '2013-05-15 14:17:31',\n",
       "   'event_name': 'Leaving sequence',\n",
       "   'event_attribute': 'startScreen, moving forward.'},\n",
       "  {'session_time': '2013-05-15 14:17:50',\n",
       "   'event_name': 'Player submitted name',\n",
       "   'event_attribute': 'Carl'},\n",
       "  {'session_time': '2013-05-15 14:17:50',\n",
       "   'event_name': 'Leaving sequence',\n",
       "   'event_attribute': 'slide2, moving forward.'},\n",
       "  {'session_time': '2013-05-15 14:17:55',\n",
       "   'event_name': 'Player submitted name',\n",
       "   'event_attribute': 'Carl'},\n",
       "  {'session_time': '2013-05-15 14:17:55',\n",
       "   'event_name': 'Leaving sequence',\n",
       "   'event_attribute': 'slide2b, moving forward.'},\n",
       "  {'session_time': '2013-05-15 14:18:34',\n",
       "   'event_name': 'Player submitted name',\n",
       "   'event_attribute': 'Carl'},\n",
       "  {'session_time': '2013-05-15 14:18:34',\n",
       "   'event_name': 'Leaving sequence',\n",
       "   'event_attribute': 'slide2c, moving forward.'},\n",
       "  {'session_time': '2013-05-15 14:20:09',\n",
       "   'event_name': 'Player submitted name',\n",
       "   'event_attribute': 'Carl'},\n",
       "  {'session_time': '2013-05-15 14:20:09',\n",
       "   'event_name': 'Leaving sequence',\n",
       "   'event_attribute': 'slide3, moving forward.'},\n",
       "  {'session_time': '2013-05-15 14:20:13',\n",
       "   'event_name': 'Player submitted name',\n",
       "   'event_attribute': 'Carl'},\n",
       "  {'session_time': '2013-05-15 14:20:13',\n",
       "   'event_name': 'Leaving sequence',\n",
       "   'event_attribute': 'slide4, moving forward.'}]}"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>session_time</th>\n",
       "      <th>event_name</th>\n",
       "      <th>event_attribute</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2013-05-15 14:17:26</td>\n",
       "      <td>Session Start</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2013-05-15 14:17:26</td>\n",
       "      <td>Leaving sequence</td>\n",
       "      <td>loadXML, moving forward.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2013-05-15 14:17:30</td>\n",
       "      <td>Player submitted name</td>\n",
       "      <td>Carl</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          session_time             event_name           event_attribute\n",
       "0  2013-05-15 14:17:26          Session Start                       NaN\n",
       "1  2013-05-15 14:17:26       Leaving sequence  loadXML, moving forward.\n",
       "2  2013-05-15 14:17:30  Player submitted name                      Carl"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(txt[\"data\"]).head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "from itertools import islice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['[5/15/2013 2:17:26 PM] Session Start\\n', '[5/15/2013 2:17:26 PM] Leaving sequence: loadXML, moving forward.\\n', '[5/15/2013 2:17:30 PM] Player submitted name: Carl\\n']\n"
     ]
    }
   ],
   "source": [
    "with open(\"./data/unstructured_example_log.txt\") as f:\n",
    "    line = f\n",
    "    print(list(islice(line,3)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "'yield' outside function (1199551098.py, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  Input \u001b[1;32mIn [48]\u001b[1;36m\u001b[0m\n\u001b[1;33m    yield line\u001b[0m\n\u001b[1;37m    ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m 'yield' outside function\n"
     ]
    }
   ],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "d83df975d3fd7e9cd2b9393f36d258bee244a85bb4d399c6c89ec18ff8dd5f85"
  },
  "kernelspec": {
   "display_name": "Python 3.8.3 ('data')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
